<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2018 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Testing a CDAP Application</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="Testing and Debugging" href="index.html" />
    <link rel="next" title="Debugging a CDAP Application" href="debugging.html" />
    <link rel="prev" title="Testing and Debugging" href="index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../building-blocks/index.html"> 抽象概念</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/applications.html"> Applications</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/datasets/index.html"> Datasets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/overview.html"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/mapreduce-programs.html"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../metadata/index.html"> 元数据</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../metadata/system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/field-lineage.html"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../pipelines/index.html"> 数据流管道</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/getting-started.html"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../pipelines/plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-pipelines.html"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 测试和调试</a><ul class="current">
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="testing-a-cdap-application">
<span id="test-cdap"></span><h1>Testing a CDAP Application<a class="headerlink" href="#testing-a-cdap-application" title="Permalink to this headline">🔗</a></h1>
<div class="section" id="strategies-in-testing-applications-test-framework">
<span id="test-framework"></span><h2>Strategies in Testing Applications: Test Framework<a class="headerlink" href="#strategies-in-testing-applications-test-framework" title="Permalink to this headline">🔗</a></h2>
<p>CDAP comes with a convenient way to unit-test your applications with CDAP’s <em>Test Framework.</em>
This framework starts an <a class="reference internal" href="../overview/modes.html#in-memory-data-application-platform"><span class="std std-ref">in-memory CDAP runtime</span></a>
and lets you deploy an application; start, stop and monitor programs; access datasets to
validate processing results; and retrieve metrics from the application.</p>
<p>The base class for such tests is <code class="docutils literal notranslate"><span class="pre">TestBase</span></code>, which is packaged
separately from the API in its own artifact because it depends on the
CDAP’s runtime classes. You can include it in your test dependencies
in one of two ways:</p>
<ul>
<li><p class="first">include all JAR files in the <code class="docutils literal notranslate"><span class="pre">lib</span></code> directory of the CDAP Sandbox installation,
or</p>
</li>
<li><p class="first">include the <code class="docutils literal notranslate"><span class="pre">cdap-unit-test</span></code> artifact in your Maven test dependencies
(as shown in the <code class="docutils literal notranslate"><span class="pre">pom.xml</span></code> file of the <a class="reference internal" href="../getting-started/sandbox/index.html#sandbox-index"><span class="std std-ref">CDAP Sandbox examples</span></a>):</p>
<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span>. . .
<span class="nt">&lt;dependency&gt;</span>
  <span class="nt">&lt;groupId&gt;</span>io.cdap.cdap<span class="nt">&lt;/groupId&gt;</span>
  <span class="nt">&lt;artifactId&gt;</span>cdap-unit-test<span class="nt">&lt;/artifactId&gt;</span>
  <span class="nt">&lt;version&gt;</span>${cdap.version}<span class="nt">&lt;/version&gt;</span>
  <span class="nt">&lt;scope&gt;</span>test<span class="nt">&lt;/scope&gt;</span>
<span class="nt">&lt;/dependency&gt;</span>
. . .
</pre></div>
</div>
</li>
</ul>
<p>Note that for building an application, you only need to include the CDAP API in your
dependencies. For testing, however, you need the CDAP run-time. To build your test case,
extend the <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class.</p>
<div class="section" id="running-tests-with-spark2">
<h3>Running Tests with Spark2<a class="headerlink" href="#running-tests-with-spark2" title="Permalink to this headline">🔗</a></h3>
<p>The <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class included in the <code class="docutils literal notranslate"><span class="pre">cdap-unit-test</span></code> dependency will run programs using
Spark1 and Scala 2.10. If you need to unit test a program that uses Spark2 and Scala 2.11,
you must remove the <code class="docutils literal notranslate"><span class="pre">cdap-unit-test</span></code> artifact from your Maven test dependencies and replace it
with <code class="docutils literal notranslate"><span class="pre">cdap-unit-test2_2.11</span></code>:</p>
<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span>. . .
<span class="nt">&lt;dependency&gt;</span>
  <span class="nt">&lt;groupId&gt;</span>io.cdap.cdap<span class="nt">&lt;/groupId&gt;</span>
  <span class="nt">&lt;artifactId&gt;</span>cdap-unit-test2_2.11<span class="nt">&lt;/artifactId&gt;</span>
  <span class="nt">&lt;version&gt;</span>${cdap.version}<span class="nt">&lt;/version&gt;</span>
  <span class="nt">&lt;scope&gt;</span>test<span class="nt">&lt;/scope&gt;</span>
<span class="nt">&lt;/dependency&gt;</span>
. . .
</pre></div>
</div>
<p>In addition, instead of extending the <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class, you must extend the <code class="docutils literal notranslate"><span class="pre">TestBaseWithSpark2</span></code> class.</p>
</div>
<div class="section" id="running-tests-from-an-ide">
<h3>Running Tests from an IDE<a class="headerlink" href="#running-tests-from-an-ide" title="Permalink to this headline">🔗</a></h3>
<p>When running tests from an IDE such IntelliJ or Eclipse, set the memory setting for the
<code class="docutils literal notranslate"><span class="pre">JUnit</span></code> tests that are run from the IDE to an increased amount of memory. We suggest
starting with:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">-Xmx1024m</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="strategies-in-testing-flows">
<span id="test-framework-strategies-flow"></span><h2>Strategies in Testing Flows<a class="headerlink" href="#strategies-in-testing-flows" title="Permalink to this headline">🔗</a></h2>
<p>Let’s write a test case for the <em>WordCount</em> example:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">WordCountTest</span> <span class="kd">extends</span> <span class="n">TestBase</span> <span class="p">{</span>
  <span class="kd">static</span> <span class="n">Type</span> <span class="n">stringMapType</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TypeToken</span><span class="o">&lt;</span><span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">String</span><span class="o">&gt;&gt;</span><span class="p">()</span> <span class="p">{</span> <span class="p">}.</span><span class="na">getType</span><span class="p">();</span>
  <span class="kd">static</span> <span class="n">Type</span> <span class="n">objectMapType</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TypeToken</span><span class="o">&lt;</span><span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">Object</span><span class="o">&gt;&gt;</span><span class="p">()</span> <span class="p">{</span> <span class="p">}.</span><span class="na">getType</span><span class="p">();</span>

  <span class="nd">@Test</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">testWordCount</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
</pre></div>
</div>
<p>The first thing we do in this test is deploy the application,
then we’ll start the flow and the service:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Deploy the application</span>
<span class="n">ApplicationManager</span> <span class="n">appManager</span> <span class="o">=</span> <span class="n">deployApplication</span><span class="p">(</span><span class="n">WordCount</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>

<span class="c1">// Start the flow and the service</span>
<span class="n">FlowManager</span> <span class="n">flowManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getFlowManager</span><span class="p">(</span><span class="s">&quot;WordCounter&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>
<span class="n">ServiceManager</span> <span class="n">serviceManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getServiceManager</span><span class="p">(</span><span class="s">&quot;RetrieveCounts&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>
<span class="n">serviceManager</span><span class="p">.</span><span class="na">waitForStatus</span><span class="p">(</span><span class="kc">true</span><span class="p">);</span>
</pre></div>
</div>
<p>This flow counts the words in the events that it receives on the “wordStream”. Before
sending data to the stream, let’s validate that we are starting with a clean state:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// validate that the wordCount table is empty, and that it has no entry for &quot;world&quot;</span>
<span class="n">DataSetManager</span><span class="o">&lt;</span><span class="n">KeyValueTable</span><span class="o">&gt;</span> <span class="n">datasetManager</span> <span class="o">=</span> <span class="n">getDataset</span><span class="p">(</span><span class="n">config</span><span class="p">.</span><span class="na">getWordCountTable</span><span class="p">());</span>
<span class="n">KeyValueTable</span> <span class="n">wordCounts</span> <span class="o">=</span> <span class="n">datasetManager</span><span class="p">.</span><span class="na">get</span><span class="p">();</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertNull</span><span class="p">(</span><span class="n">wordCounts</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="s">&quot;world&quot;</span><span class="p">));</span>
</pre></div>
</div>
<p>Note that the dataset manager implicitly starts a transaction for the dataset when it is
initialized. Under this transaction, read operations can only see changes that were
committed before the transaction was started; and any writes performed within this
transaction only become visible to other transactions after this transaction is committed.</p>
<p>This can be done by calling the dataset manager’s <code class="docutils literal notranslate"><span class="pre">flush()</span></code> method, which commits the current
transaction and starts a new one. <code class="docutils literal notranslate"><span class="pre">flush()</span></code> also needs to be called to make changes visible
to the dataset, if those changes were committed after the current transaction was started. We
will see an example for this below.</p>
<p>Now that the flow and service are running, we can send some events to the stream:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Send a few events to the stream</span>
<span class="n">StreamManager</span> <span class="n">streamManager</span> <span class="o">=</span> <span class="n">getStreamManager</span><span class="p">(</span><span class="s">&quot;words&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;hello world&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;a wonderful world&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;the world says hello&quot;</span><span class="p">);</span>
</pre></div>
</div>
<p>To wait for all events to be processed, we can get a metrics observer
for the last flowlet in the pipeline (the “word associator”) and wait for
its processed count to either reach 3 or time out after 5 seconds:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Wait for the events to be processed, or at most 5 seconds</span>
<span class="n">RuntimeMetrics</span> <span class="n">metrics</span> <span class="o">=</span> <span class="n">flowManager</span><span class="p">.</span><span class="na">getFlowletMetrics</span><span class="p">(</span><span class="s">&quot;associator&quot;</span><span class="p">);</span>
<span class="n">metrics</span><span class="p">.</span><span class="na">waitForProcessed</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">);</span>
</pre></div>
</div>
<p>Now we can start verifying that the processing was correct by reading the datasets
used by the flow. For example, we can validate the correct count for the word “world”.
Note that first we have to start a new transaction by calling <code class="docutils literal notranslate"><span class="pre">flush()</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// start a new transaction so that the &quot;wordCounts&quot; dataset sees changes made by the flow</span>
<span class="n">datasetManager</span><span class="p">.</span><span class="na">flush</span><span class="p">();</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="mi">3L</span><span class="p">,</span> <span class="n">Bytes</span><span class="p">.</span><span class="na">toLong</span><span class="p">(</span><span class="n">wordCounts</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="s">&quot;world&quot;</span><span class="p">)));</span>
</pre></div>
</div>
<p>We can also validate the processing results by obtaining a client for the service,
and then submitting queries. We’ll add a private method to the <code class="docutils literal notranslate"><span class="pre">WordCountTest</span></code>
class to help us:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">private</span> <span class="n">String</span> <span class="nf">requestService</span><span class="p">(</span><span class="n">URL</span> <span class="n">url</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">IOException</span> <span class="p">{</span>
  <span class="n">HttpURLConnection</span> <span class="n">conn</span> <span class="o">=</span> <span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">)</span> <span class="n">url</span><span class="p">.</span><span class="na">openConnection</span><span class="p">();</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">.</span><span class="na">HTTP_OK</span><span class="p">,</span> <span class="n">conn</span><span class="p">.</span><span class="na">getResponseCode</span><span class="p">());</span>
  <span class="k">try</span> <span class="p">{</span>
    <span class="k">return</span> <span class="k">new</span> <span class="n">String</span><span class="p">(</span><span class="n">ByteStreams</span><span class="p">.</span><span class="na">toByteArray</span><span class="p">(</span><span class="n">conn</span><span class="p">.</span><span class="na">getInputStream</span><span class="p">()),</span> <span class="n">Charsets</span><span class="p">.</span><span class="na">UTF_8</span><span class="p">);</span>
  <span class="p">}</span> <span class="k">finally</span> <span class="p">{</span>
    <span class="n">conn</span><span class="p">.</span><span class="na">disconnect</span><span class="p">();</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>We can then use this to query for the global statistics:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Query global statistics</span>
<span class="n">String</span> <span class="n">response</span> <span class="o">=</span> <span class="n">requestService</span><span class="p">(</span><span class="k">new</span> <span class="n">URL</span><span class="p">(</span><span class="n">serviceManager</span><span class="p">.</span><span class="na">getServiceURL</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">),</span> <span class="s">&quot;stats&quot;</span><span class="p">));</span>
</pre></div>
</div>
<p>If the query fails for any reason this method would throw an exception.
In case of success, the response is a JSON string. We must deserialize
the JSON string to verify the results:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">String</span><span class="o">&gt;</span> <span class="n">map</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Gson</span><span class="p">().</span><span class="na">fromJson</span><span class="p">(</span><span class="n">response</span><span class="p">,</span> <span class="n">stringMapType</span><span class="p">);</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;9&quot;</span><span class="p">,</span> <span class="n">map</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;totalWords&quot;</span><span class="p">));</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;6&quot;</span><span class="p">,</span> <span class="n">map</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;uniqueWords&quot;</span><span class="p">));</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(((</span><span class="kt">double</span><span class="p">)</span><span class="mi">42</span><span class="p">)</span><span class="o">/</span><span class="mi">9</span><span class="p">,</span>
  <span class="p">(</span><span class="kt">double</span><span class="p">)</span><span class="n">Double</span><span class="p">.</span><span class="na">valueOf</span><span class="p">(</span><span class="n">map</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;averageLength&quot;</span><span class="p">)),</span> <span class="mf">0.001</span><span class="p">);</span>
</pre></div>
</div>
<p>Then we ask for the statistics of one of the words in the test events.
The verification is a little more complex, because we have a nested map
as a response, and the value types in the top-level map are not uniform:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Verify some statistics for one of the words</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">requestService</span><span class="p">(</span><span class="k">new</span> <span class="n">URL</span><span class="p">(</span><span class="n">serviceManager</span><span class="p">.</span><span class="na">getServiceURL</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">),</span> <span class="s">&quot;count/world&quot;</span><span class="p">));</span>
<span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">Object</span><span class="o">&gt;</span> <span class="n">omap</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Gson</span><span class="p">().</span><span class="na">fromJson</span><span class="p">(</span><span class="n">response</span><span class="p">,</span> <span class="n">stringMapType</span><span class="p">);</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;world&quot;</span><span class="p">,</span> <span class="n">omap</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;word&quot;</span><span class="p">));</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="mf">3.0</span><span class="p">,</span> <span class="n">omap</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;count&quot;</span><span class="p">));</span>

<span class="c1">// The associations are a map within the map</span>
<span class="nd">@SuppressWarnings</span><span class="p">(</span><span class="s">&quot;unchecked&quot;</span><span class="p">)</span>
<span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">Double</span><span class="o">&gt;</span> <span class="n">assocs</span> <span class="o">=</span> <span class="p">(</span><span class="n">Map</span><span class="o">&lt;</span><span class="n">String</span><span class="p">,</span> <span class="n">Double</span><span class="o">&gt;</span><span class="p">)</span> <span class="n">omap</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;assocs&quot;</span><span class="p">);</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="mf">2.0</span><span class="p">,</span> <span class="p">(</span><span class="kt">double</span><span class="p">)</span><span class="n">assocs</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;hello&quot;</span><span class="p">),</span> <span class="mf">0.000001</span><span class="p">);</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertTrue</span><span class="p">(</span><span class="n">assocs</span><span class="p">.</span><span class="na">containsKey</span><span class="p">(</span><span class="s">&quot;hello&quot;</span><span class="p">));</span>
</pre></div>
</div>
</div>
<div class="section" id="strategies-in-testing-mapreduce-programs">
<span id="test-framework-strategies-mapreduce"></span><h2>Strategies in Testing MapReduce Programs<a class="headerlink" href="#strategies-in-testing-mapreduce-programs" title="Permalink to this headline">🔗</a></h2>
<p>In a fashion similar to <a class="reference internal" href="#strategies-in-testing-flows">Strategies in Testing Flows</a>, we can write
unit testing for MapReduce programs. Let’s write a test case for an
application that uses MapReduce.x.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">PurchaseTest</span></code> class should extend from
<code class="docutils literal notranslate"><span class="pre">TestBase</span></code> similar to <cite>Strategies in Testing Flows</cite>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">PurchaseTest</span> <span class="kd">extends</span> <span class="n">TestBase</span> <span class="p">{</span>
  <span class="nd">@Test</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">test</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">PurchaseApp</span></code> application can be deployed using the <code class="docutils literal notranslate"><span class="pre">deployApplication</span></code>
method from the <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Deploy an application</span>
<span class="n">ApplicationManager</span> <span class="n">appManager</span> <span class="o">=</span> <span class="n">deployApplication</span><span class="p">(</span><span class="n">PurchaseApp</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
</pre></div>
</div>
<p>The MapReduce reads from the <code class="docutils literal notranslate"><span class="pre">purchases</span></code> dataset. As a first
step, the data to the <code class="docutils literal notranslate"><span class="pre">purchases</span></code> should be populated by running
the <code class="docutils literal notranslate"><span class="pre">PurchaseFlow</span></code> and sending the data to the <code class="docutils literal notranslate"><span class="pre">purchaseStream</span></code>
stream:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">FlowManager</span> <span class="n">flowManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getFlowManager</span><span class="p">(</span><span class="s">&quot;PurchaseFlow&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>

<span class="c1">// Send stream events to the &quot;purchaseStream&quot; Stream</span>
<span class="n">StreamManager</span> <span class="n">streamManager</span> <span class="o">=</span> <span class="n">getStreamManager</span><span class="p">(</span><span class="s">&quot;purchaseStream&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;bob bought 3 apples for $30&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;joe bought 1 apple for $100&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;joe bought 10 pineapples for $20&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;cat bought 3 bottles for $12&quot;</span><span class="p">);</span>
<span class="n">streamManager</span><span class="p">.</span><span class="na">send</span><span class="p">(</span><span class="s">&quot;cat bought 2 pops for $14&quot;</span><span class="p">);</span>

<span class="c1">// Wait for the last flowlet to process 5 events or at most 15 seconds</span>
<span class="n">RuntimeMetrics</span> <span class="n">metrics</span> <span class="o">=</span> <span class="n">flowManager</span><span class="p">.</span><span class="na">getFlowletMetrics</span><span class="p">(</span><span class="s">&quot;collector&quot;</span><span class="p">);</span>
<span class="n">metrics</span><span class="p">.</span><span class="na">waitForProcessed</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">);</span>
</pre></div>
</div>
<p>Start the MapReduce and wait for a maximum of 60 seconds:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Start the MapReduce</span>
<span class="n">MapReduceManager</span> <span class="n">mrManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getMapReduceManager</span><span class="p">(</span><span class="s">&quot;PurchaseHistoryBuilder&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>
<span class="n">mrManager</span><span class="p">.</span><span class="na">waitForFinish</span><span class="p">(</span><span class="mi">60</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">);</span>
</pre></div>
</div>
<p>We can start verifying that the MapReduce was run correctly by
using the <code class="docutils literal notranslate"><span class="pre">PurchaseHistoryService</span></code> to retrieve a customer’s purchase history:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Start PurchaseHistoryService</span>
<span class="n">ServiceManager</span> <span class="n">purchaseHistoryServiceManager</span> <span class="o">=</span>
  <span class="n">appManager</span><span class="p">.</span><span class="na">getServiceManager</span><span class="p">(</span><span class="n">PurchaseHistoryService</span><span class="p">.</span><span class="na">SERVICE_NAME</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>

<span class="c1">// Wait for service startup</span>
<span class="n">purchaseHistoryServiceManager</span><span class="p">.</span><span class="na">waitForStatus</span><span class="p">(</span><span class="kc">true</span><span class="p">);</span>

<span class="c1">// Test service to retrieve a customer&#39;s purchase history</span>
<span class="n">URL</span> <span class="n">url</span> <span class="o">=</span> <span class="k">new</span> <span class="n">URL</span><span class="p">(</span><span class="n">purchaseHistoryServiceManager</span><span class="p">.</span><span class="na">getServiceURL</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">),</span> <span class="s">&quot;history/joe&quot;</span><span class="p">);</span>
<span class="n">HttpURLConnection</span> <span class="n">conn</span> <span class="o">=</span> <span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">)</span> <span class="n">url</span><span class="p">.</span><span class="na">openConnection</span><span class="p">();</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">.</span><span class="na">HTTP_OK</span><span class="p">,</span> <span class="n">conn</span><span class="p">.</span><span class="na">getResponseCode</span><span class="p">());</span>
<span class="n">String</span> <span class="n">historyJson</span><span class="p">;</span>
<span class="k">try</span> <span class="p">{</span>
  <span class="n">historyJson</span> <span class="o">=</span> <span class="k">new</span> <span class="n">String</span><span class="p">(</span><span class="n">ByteStreams</span><span class="p">.</span><span class="na">toByteArray</span><span class="p">(</span><span class="n">conn</span><span class="p">.</span><span class="na">getInputStream</span><span class="p">()),</span> <span class="n">Charsets</span><span class="p">.</span><span class="na">UTF_8</span><span class="p">);</span>
<span class="p">}</span> <span class="k">finally</span> <span class="p">{</span>
  <span class="n">conn</span><span class="p">.</span><span class="na">disconnect</span><span class="p">();</span>
<span class="p">}</span>
<span class="n">PurchaseHistory</span> <span class="n">history</span> <span class="o">=</span> <span class="n">GSON</span><span class="p">.</span><span class="na">fromJson</span><span class="p">(</span><span class="n">historyJson</span><span class="p">,</span> <span class="n">PurchaseHistory</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;joe&quot;</span><span class="p">,</span> <span class="n">history</span><span class="p">.</span><span class="na">getCustomer</span><span class="p">());</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">history</span><span class="p">.</span><span class="na">getPurchases</span><span class="p">().</span><span class="na">size</span><span class="p">());</span>
</pre></div>
</div>
<p>The assertion will verify that the correct result was received.</p>
</div>
<div class="section" id="strategies-in-testing-spark-programs">
<span id="test-framework-strategies-spark"></span><h2>Strategies in Testing Spark Programs<a class="headerlink" href="#strategies-in-testing-spark-programs" title="Permalink to this headline">🔗</a></h2>
<p>Let’s write a test case for an application that uses a Spark program.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">SparkPageRankTest</span></code> class should extend from
<code class="docutils literal notranslate"><span class="pre">TestBase</span></code> similar to <cite>Strategies in Testing Flows</cite>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SparkPageRankTest</span> <span class="kd">extends</span> <span class="n">TestBase</span> <span class="p">{</span>
  <span class="nd">@Test</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">test</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">SparkPageRankTest</span></code> application can be deployed using the <code class="docutils literal notranslate"><span class="pre">deployApplication</span></code>
method from the <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Deploy an application</span>
<span class="n">ApplicationManager</span> <span class="n">appManager</span> <span class="o">=</span> <span class="n">deployApplication</span><span class="p">(</span><span class="n">SparkPageRankApp</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
</pre></div>
</div>
<p>The Spark program reads from the <code class="docutils literal notranslate"><span class="pre">backlinkURLs</span></code> dataset. As a first
step, data in the <code class="docutils literal notranslate"><span class="pre">backlinkURLs</span></code> should be populated by running
the <code class="docutils literal notranslate"><span class="pre">BackLinkFlow</span></code> and sending the data to the stream <code class="docutils literal notranslate"><span class="pre">backlinkURLStream</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">FlowManager</span> <span class="n">flowManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getFlowManager</span><span class="p">(</span><span class="s">&quot;BackLinkFlow&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>
<span class="c1">// Send data to the stream</span>
<span class="n">sendData</span><span class="p">();</span>

<span class="c1">// Wait for the last flowlet to process 4 events or at most 5 seconds</span>
<span class="n">RuntimeMetrics</span> <span class="n">metrics</span> <span class="o">=</span> <span class="n">flowManager</span><span class="p">.</span><span class="na">getFlowletMetrics</span><span class="p">(</span><span class="s">&quot;reader&quot;</span><span class="p">);</span>
<span class="n">metrics</span><span class="p">.</span><span class="na">waitForProcessed</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">);</span>
</pre></div>
</div>
<p>Start the Spark program and wait for a maximum of 60 seconds:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Start the Spark program.</span>
<span class="n">SparkManager</span> <span class="n">sparkManager</span> <span class="o">=</span> <span class="n">appManager</span><span class="p">.</span><span class="na">getSparkManager</span><span class="p">(</span><span class="s">&quot;SparkPageRankProgram&quot;</span><span class="p">).</span><span class="na">start</span><span class="p">();</span>
<span class="n">sparkManager</span><span class="p">.</span><span class="na">waitForFinish</span><span class="p">(</span><span class="mi">60</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">);</span>
</pre></div>
</div>
<p>We verify that the Spark program ran correctly by
using the Ranks service to check the results:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Wait for ranks service to start</span>
<span class="n">serviceManager</span><span class="p">.</span><span class="na">waitForStatus</span><span class="p">(</span><span class="kc">true</span><span class="p">);</span>

<span class="n">String</span> <span class="n">response</span> <span class="o">=</span> <span class="n">requestService</span><span class="p">(</span><span class="k">new</span> <span class="n">URL</span><span class="p">(</span><span class="n">serviceManager</span><span class="p">.</span><span class="na">getServiceURL</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="n">TimeUnit</span><span class="p">.</span><span class="na">SECONDS</span><span class="p">),</span>
                                         <span class="s">&quot;rank?url=http://example.com/page1&quot;</span><span class="p">));</span>
<span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;14&quot;</span><span class="p">,</span> <span class="n">response</span><span class="p">);</span>
</pre></div>
</div>
<p>The assertion will verify that the correct result was received.</p>
</div>
<div class="section" id="strategies-in-testing-artifacts">
<h2>Strategies in Testing Artifacts<a class="headerlink" href="#strategies-in-testing-artifacts" title="Permalink to this headline">🔗</a></h2>
<p>The Test Framework provides methods to create and deploy JAR files as artifacts. This lets you
test the creation of multiple applications from the same artifact, as well as the use of plugin artifacts.</p>
<p>To add an artifact containing an application class:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Add the artifact for a Data Pipeline app</span>
<span class="n">addAppArtifact</span><span class="p">(</span><span class="k">new</span> <span class="n">ArtifactId</span><span class="p">(</span><span class="n">NamespaceId</span><span class="p">.</span><span class="na">DEFAULT</span><span class="p">.</span><span class="na">getNamespace</span><span class="p">(),</span> <span class="s">&quot;data-pipeline&quot;</span><span class="p">,</span> <span class="s">&quot;3.5.0&quot;</span><span class="p">),</span>
  <span class="n">DataPipelineApp</span><span class="p">.</span><span class="na">class</span><span class="p">,</span>
  <span class="n">BatchSource</span><span class="p">.</span><span class="na">class</span><span class="p">.</span><span class="na">getPackage</span><span class="p">().</span><span class="na">getName</span><span class="p">(),</span>
  <span class="n">Action</span><span class="p">.</span><span class="na">class</span><span class="p">.</span><span class="na">getPackage</span><span class="p">().</span><span class="na">getName</span><span class="p">(),</span>
  <span class="n">PipelineConfigurable</span><span class="p">.</span><span class="na">class</span><span class="p">.</span><span class="na">getPackage</span><span class="p">().</span><span class="na">getName</span><span class="p">(),</span>
  <span class="s">&quot;org.apache.avro.mapred&quot;</span><span class="p">,</span> <span class="s">&quot;org.apache.avro&quot;</span><span class="p">,</span> <span class="s">&quot;org.apache.avro.generic&quot;</span><span class="p">);</span>
</pre></div>
</div>
<p>The first argument is the <code class="docutils literal notranslate"><span class="pre">id</span></code> of the artifact; the second is the application class; and
the remainder of the arguments are packages that should be included in the
<code class="docutils literal notranslate"><span class="pre">Export-Packages</span></code> manifest attribute bundled in the JAR. The framework will trace the
dependencies of the specified application class to create a JAR with those dependencies.
This will mimic what happens when you actually build your application JAR using maven.</p>
<p>An application can then be deployed using that artifact:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Create application create request</span>
<span class="n">ETLBatchConfig</span> <span class="n">etlConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ETLBatchConfig</span><span class="p">(</span><span class="s">&quot;* * * * *&quot;</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">sink</span><span class="p">,</span> <span class="n">transformList</span><span class="p">);</span>
<span class="n">AppRequest</span><span class="o">&lt;</span><span class="n">ETLBatchConfig</span><span class="o">&gt;</span> <span class="n">appRequest</span> <span class="o">=</span> <span class="k">new</span> <span class="n">AppRequest</span><span class="o">&lt;&gt;</span><span class="p">(</span><span class="k">new</span> <span class="n">ArtifactSummary</span><span class="p">(</span><span class="s">&quot;etlbatch&quot;</span><span class="p">,</span> <span class="s">&quot;3.5.0&quot;</span><span class="p">),</span> <span class="n">etlConfig</span><span class="p">);</span>
<span class="n">ApplicationId</span> <span class="n">appId</span> <span class="o">=</span> <span class="n">NamespaceId</span><span class="p">.</span><span class="na">DEFAULT</span><span class="p">.</span><span class="na">app</span><span class="p">(</span><span class="s">&quot;KVToKV&quot;</span><span class="p">);</span>

<span class="c1">// Deploy the application</span>
<span class="n">ApplicationManager</span> <span class="n">appManager</span> <span class="o">=</span> <span class="n">deployApplication</span><span class="p">(</span><span class="n">appId</span><span class="p">,</span> <span class="n">appRequest</span><span class="p">);</span>
</pre></div>
</div>
<p>Plugins extending the artifact can also be added:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Add some test plugins</span>
<span class="n">addPluginArtifact</span><span class="p">(</span><span class="k">new</span> <span class="n">ArtifactId</span><span class="p">(</span><span class="n">NamespaceId</span><span class="p">.</span><span class="na">DEFAULT</span><span class="p">.</span><span class="na">getNamespace</span><span class="p">(),</span> <span class="s">&quot;spark-plugins&quot;</span><span class="p">,</span> <span class="s">&quot;1.0.0&quot;</span><span class="p">),</span>
                  <span class="n">APP_ARTIFACT_ID</span><span class="p">,</span>
                  <span class="n">NaiveBayesTrainer</span><span class="p">.</span><span class="na">class</span><span class="p">,</span> <span class="n">NaiveBayesClassifier</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
</pre></div>
</div>
<p>The first argument is the <code class="docutils literal notranslate"><span class="pre">id</span></code> of the plugin artifact; the second is the parent artifact
it is extending; and the remainder of the arguments are classes that should be bundled in
the JAR. The packages of all these classes are included in the <code class="docutils literal notranslate"><span class="pre">Export-Packages</span></code>
manifest attribute bundled in the JAR. When adding a plugin artifact this way, it is
important to include all classes in your plugin packages, even if they are not used in
your test case. This is to ensure that the JAR can trace all required dependencies to
correctly build the JAR.</p>
<p>The examples are taken from the <code class="docutils literal notranslate"><span class="pre">DataPipelineTest</span></code> and <code class="docutils literal notranslate"><span class="pre">HydratorTestBase</span></code> classes of CDAP pipelines.</p>
</div>
<div class="section" id="validating-test-data-with-sql">
<span id="test-framework-validating-sql"></span><h2>Validating Test Data with SQL<a class="headerlink" href="#validating-test-data-with-sql" title="Permalink to this headline">🔗</a></h2>
<p>Often the easiest way to verify that a test produced the right data is to run a SQL query—if the data sets involved
in the test case are record-scannable, as described in <a class="reference internal" href="../data-exploration/index.html#data-exploration"><span class="std std-ref">Data Exploration</span></a>.
This can be done using a JDBC connection obtained from the test base:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Obtain a JDBC connection</span>
<span class="n">Connection</span> <span class="n">connection</span> <span class="o">=</span> <span class="n">getQueryClient</span><span class="p">();</span>
<span class="k">try</span> <span class="p">{</span>
  <span class="c1">// Run a query over the dataset</span>
  <span class="n">results</span> <span class="o">=</span> <span class="n">connection</span><span class="p">.</span><span class="na">prepareStatement</span><span class="p">(</span><span class="s">&quot;SELECT key FROM mytable WHERE value = &#39;1&#39;&quot;</span><span class="p">).</span><span class="na">executeQuery</span><span class="p">();</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertTrue</span><span class="p">(</span><span class="n">results</span><span class="p">.</span><span class="na">next</span><span class="p">());</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;a&quot;</span><span class="p">,</span> <span class="n">results</span><span class="p">.</span><span class="na">getString</span><span class="p">(</span><span class="mi">1</span><span class="p">));</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertTrue</span><span class="p">(</span><span class="n">results</span><span class="p">.</span><span class="na">next</span><span class="p">());</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertEquals</span><span class="p">(</span><span class="s">&quot;c&quot;</span><span class="p">,</span> <span class="n">results</span><span class="p">.</span><span class="na">getString</span><span class="p">(</span><span class="mi">1</span><span class="p">));</span>
  <span class="n">Assert</span><span class="p">.</span><span class="na">assertFalse</span><span class="p">(</span><span class="n">results</span><span class="p">.</span><span class="na">next</span><span class="p">());</span>

<span class="p">}</span> <span class="k">finally</span> <span class="p">{</span>
  <span class="n">results</span><span class="p">.</span><span class="na">close</span><span class="p">();</span>
  <span class="n">connection</span><span class="p">.</span><span class="na">close</span><span class="p">();</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The JDBC connection does not implement the full JDBC functionality: it does not allow variable replacement and
will not allow you to make any changes to datasets. But it is sufficient to perform test validation: you can create
or prepare statements and execute queries, then iterate over the results set and validate its correctness.</p>
</div>
<div class="section" id="configuring-cdap-runtime-for-test-framework">
<span id="test-framework-configuring-cdap"></span><h2>Configuring CDAP Runtime for Test Framework<a class="headerlink" href="#configuring-cdap-runtime-for-test-framework" title="Permalink to this headline">🔗</a></h2>
<p>The <code class="docutils literal notranslate"><span class="pre">TestBase</span></code> class inherited by your test class starts an in-memory CDAP runtime before executing any test methods.
Sometimes you may need to configure the CDAP runtime to suit your specific requirements. For example, if your test
does not involve usage of SQL queries, you can turn off the explore service to reduce startup and shutdown times.</p>
<p>You alter the configurations for the CDAP runtime by applying a JUnit <code class="docutils literal notranslate"><span class="pre">&#64;ClassRule</span></code> on a <code class="docutils literal notranslate"><span class="pre">TestConfiguration</span></code>
instance. For example:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="c1">// Disable the SQL query support</span>
<span class="c1">// Set the transaction timeout to 60 seconds</span>
<span class="nd">@ClassRule</span>
<span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">TestConfiguration</span> <span class="n">CONFIG</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TestConfiguration</span><span class="p">(</span><span class="s">&quot;explore.enabled&quot;</span><span class="p">,</span> <span class="kc">false</span><span class="p">,</span>
                                                                     <span class="s">&quot;data.tx.timeout&quot;</span><span class="p">,</span> <span class="mi">60</span><span class="p">);</span>
</pre></div>
</div>
<p>Refer to the <span class="xref std std-ref">cdap-site.xml</span> for the available set of configurations used by CDAP.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Testing and Debugging" href="index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Debugging a CDAP Application" href="debugging.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>